package cn.tedu;

import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Html;
import us.codecraft.webmagic.selector.Selectable;

import java.util.List;

/**
 * 爬虫的页面解析组件
 */
public class PageJIEXI implements PageProcessor {

    private  Site site = Site.me().setTimeOut(5000).setRetryTimes(5).setSleepTime(1000);
    public void process(Page page) {

        //获取页面html对象
//        System.out.println(page);
        //url
        Html html = page.getHtml();
        List<String> all = html.links().regex("http://www.dangjian.com/[\\w/.stml]+").all();
        //继续自动爬取
        page.addTargetRequests(all);
        //输出 标题
        System.out.println(html.getDocument().title());
        //获取当前的url    代码
        String url = page.getUrl().all().get(0);
        System.out.println(url);
        //解析页面内容   url  包含   .shtml
        if(url.contains("shtml")){
         //解析页面内容
            Document document = html.getDocument();
            //获取标题类 title_tex
            String title_tex = document.getElementById("title_tex").text();
            //发表时间
            String time_tex = document.getElementById("time_tex").text();
            //来源
            String time_ly = document.getElementById("time_ly").text();
            //内容
            Elements elementsByClass = document.getElementsByClass("lx-box1");
            //作者
            String bj = document.getElementsByClass("bj").text();
            //类型
            String currChnlCls = document.getElementsByClass("CurrChnlCls").text();
            //把数据发送到爬虫的数据持久化组件中
            page.putField("title_tex",title_tex);
            page.putField("time_tex",time_tex);
            page.putField("time_ly",time_ly);
            page.putField("elementsByClass",elementsByClass);
            page.putField("bj",bj);
            page.putField("currChnlCls",currChnlCls);//类型


        }else {
            //不解析
            System.out.println("不是页面信息");
        }



    }

    public Site getSite() {
        return site;
    }
}
